package com.devilvan.jsoup;

import com.alibaba.fastjson.JSONObject;
import lombok.Data;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.jsoup.nodes.Element;
import org.junit.jupiter.api.Test;

import java.io.*;
import java.net.*;
import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

@Data
public class Crawl {
    private String url;
    private Integer pageCount;
    private Map<String, String> header;
    private Map<String, String> crawlConnectHeader;
    private String path;
    private String fileName;

    //    private final static String DOMAIN_ADDRESS = "ieeexplore-ieee-org.mxg.80599.net";
    private final static String DOMAIN_ADDRESS = "xploreqa.ieee.org";


    @Test
    public void crawlReal() {
        Map<String, String> crawlHeader = new HashMap<>();
        crawlHeader.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
        crawlHeader.put("Accept-Encoding", "zip, deflate, br");
        crawlHeader.put("Connection", "keep-alive");
        crawlHeader.put("Host", DOMAIN_ADDRESS);
        crawlHeader.put("Cookie", "s_ecid=MCMID%7C59769930633809081770330147941875448076; __gads=ID=66c624d06eca9f39:T=1634886630:S=ALNI_MboG4Kn_IHeStBsBQ53vvlDPpj8gw; _gcl_au=1.1.1838112927.1634886843; _ga=GA1.2.1809138230.1634886844; LT=XPLLG_XPL_2021_SUB_SignIn_Purchase; _uetvid=ae7548f0330711ec9244015150a7f460; _mkto_trk=id:682-UPB-550&token:_mch-ieee.org-1634886863556-57954; s_fid=1AC325EE049891FA-3EDCE0DCA9932DB4; fp=30a5258d9b8c32e6c9d7b03a7c32ec2d; WLSESSION=250241676.20480.0000; JSESSIONID=4S3gY2V0eiJxLOhsABl2Dj02KvXKFCnQCFkEcJwHYGyhwqBr1yQL!-211755316; ipCheck=183.238.255.190; ERIGHTS=3KgcADheY8QhgDoLz3JPNQJx2Farx2BGP0Ja*4oUBsVJaGSO7YAGYgE9zugx3Dx3D-18x2d2CJ01FdpHj7soix2BefrHXeAx3Dx3Di1ewyzpCHHydODWVkat0swx3Dx3D-yH3qDVeUVf2x2FAthQq1haUgx3Dx3D-fW60WTfxxQJcSXcRE9Y6V9wx3Dx3D; TS017ca1d7=012521a89b8b64efd54dd16c772d466b4d0f917de2f371f2fee6f3743883a0b62a0cb014b63ac78b5c29e50aa80b06447b7537ae62; ipList=183.238.255.190; AMCVS_8E929CC25A1FB2B30A495C97%40AdobeOrg=1; AMCV_8E929CC25A1FB2B30A495C97%40AdobeOrg=1687686476%7CMCIDTS%7C18923%7CMCMID%7C59769930633809081770330147941875448076%7CMCAAMLH-1636457002%7C11%7CMCAAMB-1636457002%7CRKhpRz8krg2tLO6pguXWp5olkAcUniQYPHaMWWgdJ3xzPWQmdj0y%7CMCOPTOUT-1635859402s%7CNONE%7CMCAID%7CNONE%7CvVersion%7C3.0.0; s_cc=true; utag_main=v_id:017ca6d6242500aefe028f4a20100d07f005b07700aee$_sn:9$_se:2$_ss:0$_st:1635854016273$vapi_domain:ieee.org$ses_id:1635852201646%3Bexp-session$_pn:2%3Bexp-session; s_sq=ieeexplore.dev%3D%2526c.%2526a.%2526activitymap.%2526page%253DSearch%252520Results%2526link%253DOpening%252520the%252520Deep%252520Pandora%252520Box%25253A%252520Explainable%252520Traffic%252520Classification%2526region%253D9162704%2526pageIDType%253D1%2526.activitymap%2526.a%2526.c%2526pid%253DSearch%252520Results%2526pidt%253D1%2526oid%253Dhttps%25253A%25252F%25252Fxploreqa.ieee.org%25252Fdocument%25252F9162704%25252F%2526ot%253DA; xpluserinfo=eyJpc0luc3QiOiJ0cnVlIiwiaW5zdE5hbWUiOiJORUQgVU5JViBPRiBFTkdJTkVFUklORyBBTkQgVEVDSE5PTE9HWSIsInByb2R1Y3RzIjoiSUVMfFZERXxOT0tJQSBCRUxMIExBQlN8In0=; seqId=39887; TS092f95c8027=08c7abc71bab2000916999e74edaa979bf2eaedd17ac30b5c429e35c4dc3b3c930a318c86fe7790e08a892add71130000cd511a687dd51d5eb987dc2f30f002552a3045a1214b864178d74e1fa18359db2d950bbb71aa97aa028cb3fba4960a3");
        try {
            Element bodyElement = CrawlUtil.getBodyElement("https://xploreqa.ieee.org/stamp/stamp.jsp?tp=&arnumber=9162743", crawlHeader);
            String pdfPath = bodyElement.select("iframe").attr("src");
            System.out.println("pdfPath: " + pdfPath);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }

    public static void main(String[] args) throws InterruptedException {
//        mergeFiles();
//        mergeFiles2();

        Crawl crawl = new Crawl();
        Map<String, String> headerMap = new HashMap<>();
        headerMap.put("Accept", "application/json, text/plain, */*");
        headerMap.put("Accept-Encoding", "gzip, deflate, br");
        headerMap.put("Referer", "https://" + DOMAIN_ADDRESS + "/search/searchresult.jsp?queryText=infocom%2021");
        headerMap.put("Host", DOMAIN_ADDRESS);
        headerMap.put("Origin", "https://" + DOMAIN_ADDRESS);
        headerMap.put("Connection", "Keep-Alive");
        headerMap.put("Content-Type", "application/json; charset=utf-8");
        headerMap.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36 SE 2.X MetaSr 1.0");
        headerMap.put("Cookie", "JSESSIONID=3KentlO3boinQ96X5ElT8fpVEHOqVPYeK125XlmxiTHuJILH66Ji!-723651570; ipCheck=240e:379:20d0:b400:4880:e624:3bbe:f6ac; WLSESSIONAZ=203579914.20480.0000; TS01109a32=012933e659476ca0542486c3098b99ce7627df65516a3a8a1efdab3880dcced1417071a96e4b29133d2941b56bfad5746a443e69e3; fp=da59c49efece9fd477186dc0aac4d890; AMCVS_8E929CC25A1FB2B30A495C97%40AdobeOrg=1; __gads=ID=e15fce292d19d926:T=1634901349:S=ALNI_MZkNF25G1Bv-GUtjZrWHW01iRVXfw; s_ecid=MCMID%7C06651062624548266401510636503122186243; AMCV_8E929CC25A1FB2B30A495C97%40AdobeOrg=1687686476%7CMCIDTS%7C18923%7CMCMID%7C06651062624548266401510636503122186243%7CMCAAMLH-1635506149%7C11%7CMCAAMB-1635506149%7CRKhpRz8krg2tLO6pguXWp5olkAcUniQYPHaMWWgdJ3xzPWQmdj0y%7CMCOPTOUT-1634908549s%7CNONE%7CMCAID%7CNONE%7CvVersion%7C3.0.0; s_cc=true; utag_main=v_id:017ca7b6c8b000175f255b62f7bd0f07f005b07700bd0$_sn:1$_se:11$_ss:0$_st:1634905790889$ses_id:1634901346482%3Bexp-session$_pn:8%3Bexp-session$vapi_domain:ieee.org; TS01dcb973=012933e6590bbdf41537244885ac0e7f5d6dd77844a5300a59b9debe68af637090f2c6eb47af0f59b908795b91d0ab70485b8fab27");
        Map<String, String> crawlHeader = new HashMap<>();
        crawlHeader.put("Accept", "application/json, text/plain, */*");
        crawlHeader.put("Accept-Encoding", "gzip, deflate, br");
        crawlHeader.put("Accept-Language", "zh-CN,zh;q=0.9");
        crawlHeader.put("Cache-Control", "no-cache");
        crawlHeader.put("Connection", "keep-alive");
        crawlHeader.put("Cookie", "JSESSIONID=3KentlO3boinQ96X5ElT8fpVEHOqVPYeK125XlmxiTHuJILH66Ji!-723651570; ipCheck=240e:379:20d0:b400:4880:e624:3bbe:f6ac; WLSESSIONAZ=203579914.20480.0000; TS01109a32=012933e659476ca0542486c3098b99ce7627df65516a3a8a1efdab3880dcced1417071a96e4b29133d2941b56bfad5746a443e69e3; fp=da59c49efece9fd477186dc0aac4d890; AMCVS_8E929CC25A1FB2B30A495C97%40AdobeOrg=1; __gads=ID=e15fce292d19d926:T=1634901349:S=ALNI_MZkNF25G1Bv-GUtjZrWHW01iRVXfw; s_ecid=MCMID%7C06651062624548266401510636503122186243; AMCV_8E929CC25A1FB2B30A495C97%40AdobeOrg=1687686476%7CMCIDTS%7C18923%7CMCMID%7C06651062624548266401510636503122186243%7CMCAAMLH-1635506149%7C11%7CMCAAMB-1635506149%7CRKhpRz8krg2tLO6pguXWp5olkAcUniQYPHaMWWgdJ3xzPWQmdj0y%7CMCOPTOUT-1634908549s%7CNONE%7CMCAID%7CNONE%7CvVersion%7C3.0.0; s_cc=true; TS01dcb973=012933e6599bc94d770c725d85443df35150d388fbe86e931cbd9959aae4bae947b0b0ab2beb34db76a7490a3ea8a554e77da3ae7d; utag_main=v_id:017ca7b6c8b000175f255b62f7bd0f07f005b07700bd0$_sn:1$_se:6$_ss:0$_st:1634905375903$ses_id:1634901346482%3Bexp-session$_pn:4%3Bexp-session$vapi_domain:ieee.org");
        crawlHeader.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36 SE 2.X MetaSr 1.0");


        crawl.setUrl("https://" + DOMAIN_ADDRESS + "/rest/search");
        crawl.setPageCount(20);
        crawl.setHeader(headerMap);
        crawl.setCrawlConnectHeader(crawlHeader);
        SearchParam searchParam = new SearchParam().
                setQueryText("infocom 2021").
                setHighlight(true).
                setReturnFacets(new String[]{"ALL"}).
                setReturnType("SEARCH");
        crawl.searchIeee(crawl.getUrl(), crawl.getPageCount(), searchParam);
    }

    public void searchIeee(String url, Integer pageCount, Object jsonObject) throws InterruptedException {
        CountDownLatch countDownLatch = new CountDownLatch(pageCount);
        for (int i = 1; i <= pageCount; i++) {
            int finalI = i;
            SearchParam searchParam = (SearchParam) jsonObject;
            searchParam.setPageNumber(String.valueOf(finalI));
            String json = JSONObject.toJSONString(searchParam);
            new Thread(() -> {
                try {
                    System.out.println(json);
                    crawlIeee(url, json, finalI);
                } catch (Exception e) {
                    e.printStackTrace();
                } finally {
                    countDownLatch.countDown();
                }
            }).start();
        }
        countDownLatch.await();
    }

    public void crawlIeee(String url, String json, int i) {
        try {
            String jsonResult = HttpClientUtil.httpGetByJson(url, json, header);
            IEEEObjectDto ieeeObjectDto = JSONObject.parseObject(jsonResult, IEEEObjectDto.class);
            FileWriter fileWriter = new FileWriter("D:\\Mine\\ieee2\\" + i + ".txt");
            for (Record record : ieeeObjectDto.getRecords()) {
                CrawlUtil.writeLine(fileWriter, "英文标题：", record.getArticleTitle());
//                CrawlUtil.writeLine(fileWriter, "中文标题", CrawlUtil.googleTranslate("en", "zh-CN", record.getArticleTitle()));
                List<String> preferredNames = record.getAuthors().stream().map(Author::getPreferredName).collect(Collectors.toList());
                CrawlUtil.writeLine(fileWriter, "作者：", preferredNames);
                // 扒取完整的摘要
                String documentLink = "https://" + DOMAIN_ADDRESS + record.getDocumentLink();
                Element bodyElement = CrawlUtil.getBodyElement(documentLink, crawlConnectHeader);
                String content = bodyElement.parent().select("meta[property='twitter:description']").attr("content");
                CrawlUtil.writeLine(fileWriter, "英文摘要", content);
//                CrawlUtil.writeLine(fileWriter, "中文摘要", CrawlUtil.googleTranslate("en", "zh-CN", content));
                CrawlUtil.writeLine(fileWriter, "发布时间：", record.getPublicationDate());

                // 开始下载pdf
//                https://xploreqa.ieee.org/stamp/stamp.jsp?tp=&arnumber=9197854
                // 爬取pdf下载路径
                String pdfLink = "https://" + DOMAIN_ADDRESS + "/stamp/stamp.jsp?tp=&arnumber=" + record.getArticleNumber();
                Map<String, String> crawlHeader = new HashMap<>();
                crawlHeader.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
                crawlHeader.put("Accept-Encoding", "zip, deflate, br");
                crawlHeader.put("Connection", "keep-alive");
                crawlHeader.put("Host", DOMAIN_ADDRESS);
                crawlHeader.put("Cookie", "utagdb=true; s_ecid=MCMID%7C59769930633809081770330147941875448076; __gads=ID=66c624d06eca9f39:T=1634886630:S=ALNI_MboG4Kn_IHeStBsBQ53vvlDPpj8gw; _gcl_au=1.1.1838112927.1634886843; _ga=GA1.2.1809138230.1634886844; LT=XPLLG_XPL_2021_SUB_SignIn_Purchase; _uetvid=ae7548f0330711ec9244015150a7f460; _mkto_trk=id:682-UPB-550&token:_mch-ieee.org-1634886863556-57954; s_fid=1AC325EE049891FA-3EDCE0DCA9932DB4; fp=30a5258d9b8c32e6c9d7b03a7c32ec2d; WLSESSION=250241676.20480.0000; JSESSIONID=4S3gY2V0eiJxLOhsABl2Dj02KvXKFCnQCFkEcJwHYGyhwqBr1yQL!-211755316; ipCheck=183.238.255.190; ERIGHTS=3KgcADheY8QhgDoLz3JPNQJx2Farx2BGP0Ja*4oUBsVJaGSO7YAGYgE9zugx3Dx3D-18x2d2CJ01FdpHj7soix2BefrHXeAx3Dx3Di1ewyzpCHHydODWVkat0swx3Dx3D-yH3qDVeUVf2x2FAthQq1haUgx3Dx3D-fW60WTfxxQJcSXcRE9Y6V9wx3Dx3D; ipList=183.238.255.190; AMCVS_8E929CC25A1FB2B30A495C97%40AdobeOrg=1; AMCV_8E929CC25A1FB2B30A495C97%40AdobeOrg=1687686476%7CMCIDTS%7C18923%7CMCMID%7C59769930633809081770330147941875448076%7CMCAAMLH-1636457002%7C11%7CMCAAMB-1636457002%7CRKhpRz8krg2tLO6pguXWp5olkAcUniQYPHaMWWgdJ3xzPWQmdj0y%7CMCOPTOUT-1635859402s%7CNONE%7CMCAID%7CNONE%7CvVersion%7C3.0.0; s_cc=true; TS017ca1d7=012521a89baba18ff02d5b473325b0eca209a204d077c02b68ba93bbe0713b9e85af7eccbf89d489a7577496d6bab93d0ea0779d4d; s_sq=%5B%5BB%5D%5D; utag_main=v_id:017ca6d6242500aefe028f4a20100d07f005b07700aee$_sn:9$_se:6$_ss:0$_st:1635854760591$vapi_domain:ieee.org$ses_id:1635852201646%3Bexp-session$_pn:6%3Bexp-session; xpluserinfo=eyJpc0luc3QiOiJ0cnVlIiwiaW5zdE5hbWUiOiJORUQgVU5JViBPRiBFTkdJTkVFUklORyBBTkQgVEVDSE5PTE9HWSIsInByb2R1Y3RzIjoiSUVMfFZERXxOT0tJQSBCRUxMIExBQlN8In0=; seqId=39887; TS092f95c8027=08c7abc71bab2000309c7d8d10b6d6e7bea4f366c0345f898fd0b75851c6cc2c4b485c065e5d21ca082c339478113000e4e839dbd1132f04107665cd674a2011b88033c4bbd68f7e0db018479d7f481a2a2017b81feba9db58795da971c604f3");
                Element pdfBodyElement = CrawlUtil.getBodyElement(pdfLink, crawlHeader);
//                String data = pdfBodyElement.data();
//                String pattern = "/iel7\\S+(.pdf)";
//                String pdfPath = CrawlUtil.matchRegularExp(data, pattern);
                String pdfPath = pdfBodyElement.select("iframe").attr("src");
                System.out.println("pdfPath: " + pdfPath);
                if (EmptyUtil.isNotEmpty(pdfPath)) record.setPdfPath(pdfPath);
                if (EmptyUtil.isNotEmpty(record.getPdfPath()) &&
                        EmptyUtil.isNotEmpty(record.getArticleNumber()) &&
                        EmptyUtil.isNotEmpty(record.getArticleTitle()) &&
                        EmptyUtil.isNotEmpty(record.getPublicationLink()) &&
                        EmptyUtil.isNotEmpty(record.getPublicationNumber())) {
                    String articleTitle = replace(record.getArticleTitle());
                    String urlString = record.getPdfPath();
                    CrawlUtil.writeLine(fileWriter, "PDF路径：", urlString);
                    CrawlUtil.writeSplitLine(fileWriter);
//                    System.out.println("urlString: " + urlString);
                    downloadUrlPdf(urlString, articleTitle, record.getPublicationLink());
                }
            }
            fileWriter.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }


    public static void downloadUrlPdf(String urlString, String fileName, String publicationLink) {
        InputStream dataInputStream = null;
        OutputStream outputStream = null;
        try {
            URL url = new URL(urlString);
            HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
            urlConnection.setRequestProperty("Connection", "keep-alive");
            urlConnection.setRequestProperty("Content-Type", "application/pdf");
            urlConnection.setRequestProperty("Referer", "https://" + DOMAIN_ADDRESS + publicationLink);
            urlConnection.setRequestProperty("Content-Disposition", "inline; filename=" + fileName + ".pdf");
            urlConnection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36 SE 2.X MetaSr 1.0");
            urlConnection.setRequestProperty("Host", DOMAIN_ADDRESS);
            urlConnection.setRequestProperty("Cookie","utagdb=true; s_ecid=MCMID%7C59769930633809081770330147941875448076; __gads=ID=66c624d06eca9f39:T=1634886630:S=ALNI_MboG4Kn_IHeStBsBQ53vvlDPpj8gw; _gcl_au=1.1.1838112927.1634886843; _ga=GA1.2.1809138230.1634886844; LT=XPLLG_XPL_2021_SUB_SignIn_Purchase; _uetvid=ae7548f0330711ec9244015150a7f460; _mkto_trk=id:682-UPB-550&token:_mch-ieee.org-1634886863556-57954; s_fid=1AC325EE049891FA-3EDCE0DCA9932DB4; fp=30a5258d9b8c32e6c9d7b03a7c32ec2d; WLSESSION=250241676.20480.0000; JSESSIONID=4S3gY2V0eiJxLOhsABl2Dj02KvXKFCnQCFkEcJwHYGyhwqBr1yQL!-211755316; ipCheck=183.238.255.190; ERIGHTS=3KgcADheY8QhgDoLz3JPNQJx2Farx2BGP0Ja*4oUBsVJaGSO7YAGYgE9zugx3Dx3D-18x2d2CJ01FdpHj7soix2BefrHXeAx3Dx3Di1ewyzpCHHydODWVkat0swx3Dx3D-yH3qDVeUVf2x2FAthQq1haUgx3Dx3D-fW60WTfxxQJcSXcRE9Y6V9wx3Dx3D; ipList=183.238.255.190; AMCVS_8E929CC25A1FB2B30A495C97%40AdobeOrg=1; AMCV_8E929CC25A1FB2B30A495C97%40AdobeOrg=1687686476%7CMCIDTS%7C18923%7CMCMID%7C59769930633809081770330147941875448076%7CMCAAMLH-1636457002%7C11%7CMCAAMB-1636457002%7CRKhpRz8krg2tLO6pguXWp5olkAcUniQYPHaMWWgdJ3xzPWQmdj0y%7CMCOPTOUT-1635859402s%7CNONE%7CMCAID%7CNONE%7CvVersion%7C3.0.0; s_cc=true; TS017ca1d7=012521a89baba18ff02d5b473325b0eca209a204d077c02b68ba93bbe0713b9e85af7eccbf89d489a7577496d6bab93d0ea0779d4d; s_sq=%5B%5BB%5D%5D; utag_main=v_id:017ca6d6242500aefe028f4a20100d07f005b07700aee$_sn:9$_se:6$_ss:0$_st:1635854760591$vapi_domain:ieee.org$ses_id:1635852201646%3Bexp-session$_pn:6%3Bexp-session; xpluserinfo=eyJpc0luc3QiOiJ0cnVlIiwiaW5zdE5hbWUiOiJORUQgVU5JViBPRiBFTkdJTkVFUklORyBBTkQgVEVDSE5PTE9HWSIsInByb2R1Y3RzIjoiSUVMfFZERXxOT0tJQSBCRUxMIExBQlN8In0=; seqId=39887; TS092f95c8027=08c7abc71bab2000309c7d8d10b6d6e7bea4f366c0345f898fd0b75851c6cc2c4b485c065e5d21ca082c339478113000e4e839dbd1132f04107665cd674a2011b88033c4bbd68f7e0db018479d7f481a2a2017b81feba9db58795da971c604f3");
            urlConnection.setRequestMethod("GET");
            TimeUnit.MILLISECONDS.sleep(500);
            urlConnection.connect();
            dataInputStream = urlConnection.getInputStream();
            byte[] bytes = readInputStream(dataInputStream);
            //            File file = new File("D:\\Mine\\ieee\\pdf");
            //            if (!file.exists()) file.mkdirs();
            outputStream = new FileOutputStream("D:\\Mine\\ieee2\\pdf" + "\\" + fileName + ".pdf");
            outputStream.write(bytes);
        } catch (IOException | InterruptedException e) {
            e.printStackTrace();
        } finally {
            try {
                if (null != dataInputStream) dataInputStream.close();
                if (null != outputStream) outputStream.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

    public static String replace(String re) {
        return re.replaceAll("[:]", "").replaceAll("[-\\/:*?\"<>| ]", "_");
    }

    /**
     * 从输入流中获取字节数组
     *
     * @param inputStream
     * @return
     * @throws IOException
     */
    public static byte[] readInputStream(InputStream inputStream) throws IOException {
        byte[] buffer = new byte[1024];
        int len = 0;
        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        while ((len = inputStream.read(buffer)) != -1) {
            bos.write(buffer, 0, len);
        }
        bos.close();
        return bos.toByteArray();
    }

    public static String subString(String sub) {
        return !"".equals(sub) && null != sub ? sub.substring(sub.length() - 8, sub.length() - 1) : null;
    }

    public static void mergeFiles() {
        try {
            FileSystem fileSystem = FileSystem.get(new Configuration());
            // 创建一个文件，存放合并后的结果
            FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("D:\\Mine\\ieee\\mergeResult.txt"));
            LocalFileSystem localFileSystem = FileSystem.getLocal(new Configuration());
            FileStatus[] fileStatuses = localFileSystem.listStatus(new Path("D:\\Mine\\ieee"));
            for (FileStatus fileStatus : fileStatuses) {
                //获取每一个本地的文件路径
                Path path = fileStatus.getPath();
                //读取本地小文件
                FSDataInputStream fsDataInputStream = localFileSystem.open(path);
                IOUtils.copy(fsDataInputStream, fsDataOutputStream);
                IOUtils.closeQuietly(fsDataInputStream);
            }
            localFileSystem.close();
            IOUtils.closeQuietly(fsDataOutputStream);
            fileSystem.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public static void mergeFiles2() {
        try {
            String resultPath = "D:\\Mine\\ieee\\2021.txt";
            FileWriter fileWriter = new FileWriter(resultPath);
            int len = 20;
            for (int i = 1; i <= len; i++) {
                //读取本地小文件
                FileReader fileReader = new FileReader("D:\\Mine\\ieee" + "\\" + i + ".txt");
                IOUtils.copy(fileReader, fileWriter);
                fileReader.close();
            }
            fileWriter.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

}
